#include "InvertIndexLib.h"

#include <fstream>
#include <sstream>
#include <cmath>
#include <codecvt>
#include <iomanip>

using std::ifstream;
using std::istringstream;
using std::ofstream;


InvertIndexLib::InvertIndexLib(const string ripepageLibFileName, 
                               const string offsetLibFileName, 
                               const string stopWordFileName)
: _ripepageLibFileName(ripepageLibFileName)
, _offsetLibFileName(offsetLibFileName)
, _stopWordFileName(stopWordFileName)
, _pageNum(0) {}
    
void InvertIndexLib::create() {
    makeOffsetMap();
    /* showMap(); */
    makeStopWordVec();
    /* showVec(); */
    ifstream ifs(_ripepageLibFileName);
    if(!ifs) {
        perror("InvertIndexLib::create::ifs");
        return;
    }
    vector<string> res;
    int id;
    int length;
    //拿到的是其中一篇文章
    for(auto &elem : _offsetLib) {
        ++_pageNum;
        id = elem.first;
        int begin = elem.second.first;
        length = elem.second.second;
        ifs.seekg(begin, std::ios::beg);
        char *buf = new char[length + 1]();
        ifs.read(buf, length);
        buf[length] = '\0';//将文章读进buf

        string paper = buf;

        delete [] buf;

        string paperProcess = "";
        for(char ch : paper) {
            if(!std::isspace(ch) && !std::ispunct(ch)) {
                paperProcess += ch;
            }
        }
        res.clear();
        res = _wordSeg.cut(paperProcess);

        makeWordWeiMap(res, length, id);//先构成需要的元素，但将权重初始化为0

        /* set<string> word(res.begin(), res.end()); */
        
        //for(auto &elem : word) {
        //    if(isalpha(elem[0])) continue;
        //    auto it = _stopWord.find(elem);
        //    if(it == _stopWord.end()) {
        //        word.erase(elem);
        //    }
        //}
        
        _helpMap[id] = set<string>(res.begin(), res.end());
        /* _helpMap[id] = word; */
    }
    /* showMap(); */
    helpMakeWordWeiMap();//在循环外面计算权重，降低时间复杂度
    //走出上面的循环意味着已经把全部的文章读进了_wordWei
    /* makeWordWeiMap2();//计算权重往map里加 */
    createInvertLib(); 
}


//单词，次数，权重设置为0
void InvertIndexLib::makeWordWeiMap(const vector<string> &word, int N, int id) {
    //插入词语出现的次数
    /* std::cout << "word.size() = " << word.size() << std::endl; */
    /* std::cout << "_stopWord.size() = " << _stopWord.size() << std::endl; */
    //for(auto &elem : word) {
    //    for(auto &stop : _stopWord) {
    //        if(elem != stop) {
    //            ++_wordWei[elem][id].first;
    //            _wordWei[elem][id].second = 0;
    //        }
    //    }
    //}
    

    //去停用词,用map降低比对速度
    for(auto &elem : word) {
        if(isalpha(elem[0])) continue;
        auto it = _stopWord.find(elem);
        if(it == _stopWord.end()) {
            ++_wordWei[elem][id].first;
            _wordWei[elem][id].second = 0.000000;
            ++_totalTimes[elem];//短语出现的总次数
        }
    }
}

//map<int, map<string, int>> _helpMap村的文章id和cppjieba切词之后的结果
void InvertIndexLib::helpMakeWordWeiMap() {
    std::cout << "文章篇数：" << _pageNum << std::endl;
    std::cout << "短语个数：" << _wordWei.size() << std::endl;

    for(int id = 1; id < _pageNum + 1; ++id) {
        double wAll = 0.000000;
        for(auto &elem : _helpMap[id]) {
            int TF = _wordWei[elem][id].first;
            int DF = _totalTimes[elem];
            double IDF = std::log2(_pageNum / (DF + 1));
            double w =TF * IDF;
            wAll += w * w;
        }
    }
    for(int id = 1; id < _pageNum + 1; ++id) {
        for(auto &elem : _helpMap[id]) {
            int TF = _wordWei[elem][id].first;
            int DF = _totalTimes[elem];
            double IDF = std::log2(_pageNum / (DF + 1));
            double w =TF * IDF;
            _wordWei[elem][id].second = w;
        }
    }
}


void InvertIndexLib::createInvertLib() {
    ofstream ofs("invertIndex.dat", std::ios::app);
    if(!ofs) {
        perror("InvertIndexLib::createInvertLib");
        return;
    }
    
    //for(auto &elem : _wordWei) {
    //    if(!isalpha(elem.first[0])) {
    //        ofs << elem.first << " ";//词语
    //        for(auto &it : elem.second) {
    //            ofs << it.first << " " << std::fixed << std::setprecision(6) << it.second.second << " ";//文章id和权重
    //        }
    //        ofs << std::endl;
    //    } 
    //}
    
    int flag = 0;
    for(auto &elem : _wordWei) {
        int times = 0;
        if(!isalpha(elem.first[0])) {
            for(auto &it : elem.second) {
                if(it.second.second != 0.000000) {
                    flag = 1;
                    if(!times) {
                        times = 1;
                        ofs << elem.first << " ";//词语

                    }
                    ofs << it.first << " " << it.second.second << " ";//文章id和权重
                }
            }
            if(flag) {
                ofs << std::endl;
            }
            times = 0;
        } 
    }
}

void InvertIndexLib::makeOffsetMap() {
    ifstream ifs(_offsetLibFileName);
    if(!ifs) {
        perror("InvertIndexLib::makeOffsetMap");
        return;
    }
    string line;
    while(getline(ifs, line)) {
        int id;
        int begin;
        int length;
        istringstream iss(line);
        iss >> id;
        iss >> begin;
        iss >> length;
        _offsetLib.insert({id, std::make_pair(begin, length)}); 
    }
    ifs.close();
}

void InvertIndexLib::makeStopWordVec() {
    ifstream ifs(_stopWordFileName);
    if(!ifs) {
        perror("InvertIndexLib::makeStopWordVec");
        return;
    }
    string line;
    while(getline(ifs, line)) {
        istringstream iss(line);
        string word;
        while(iss >> word) {
            _stopWord[word];
        }
    }
    ifs.close();
}


void InvertIndexLib::showMap() {
    /* for(auto &elem : _offsetLib) { */
    /*     std::cout << elem.first << " " << elem.second.first << " " << elem.second.second << std::endl; */
    /* } */
    for(auto &elem : _totalTimes) {
        std::cout << elem.first << " " << elem.second << std::endl;
    }
}

void InvertIndexLib::showVec() {
    for(auto &elem : _stopWord) {
        std::cout << elem << std::endl;
    }
}
